{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "76cadc53-5ac7-48d9-ae12-d900eede6f14",
   "metadata": {},
   "source": [
    "# 模型推理 - 使用 QLoRA 微调后的 ChatGLM-6B"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "933ea591-747f-45a9-bc85-d29c5fc88d7a",
   "metadata": {},
   "source": [
    "# 导入使用的组件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0042bbb0-0457-446d-90b2-8345c490cb02",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import AutoModel, AutoTokenizer, BitsAndBytesConfig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e23183fa-0605-46e9-9d2f-559c69bf27da",
   "metadata": {},
   "source": [
    "# 全局参数定义"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "60f78984-97b6-4e0b-909b-9d96453d4f90",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 微调前模型ID\n",
    "model_name_or_path = 'THUDM/chatglm3-6b'\n",
    "# 微调后模型本地路径\n",
    "\n",
    "_compute_dtype_map = {\n",
    "    'fp32': torch.float32,\n",
    "    'fp16': torch.float16,\n",
    "    'bf16': torch.bfloat16\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3367348e-02bb-4de2-99bf-30f3114f44c6",
   "metadata": {},
   "source": [
    "# 测试文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e2801b9d-3e3e-4fda-bce5-4e6842a0859c",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = \"解释下乾卦是什么？\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1517259-5389-4a9f-ba9b-911fd5012369",
   "metadata": {},
   "source": [
    "# 量化配置，使用微调前的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "94d4102b-e078-43b0-bd52-0d0fb57c37a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/lm_ai_learn/lib/python3.10/site-packages/huggingface_hub/file_download.py:945: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "/root/miniconda3/envs/lm_ai_learn/lib/python3.10/site-packages/huggingface_hub/file_download.py:945: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04d48d4aac094ca29d0ba3b59b31fbe2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/lm_ai_learn/lib/python3.10/site-packages/huggingface_hub/file_download.py:945: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# QLoRA 量化配置\n",
    "q_config = BitsAndBytesConfig(load_in_4bit=True,\n",
    "                              bnb_4bit_quant_type='nf4',\n",
    "                              bnb_4bit_use_double_quant=True,\n",
    "                              bnb_4bit_compute_dtype=_compute_dtype_map['bf16'])\n",
    "\n",
    "# 加载量化后模型(与微调的 revision 保持一致）\n",
    "base_model = AutoModel.from_pretrained(model_name_or_path,\n",
    "                                      quantization_config=q_config,\n",
    "                                      device_map='auto',\n",
    "                                      trust_remote_code=True,\n",
    "                                      revision='b098244')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1355a4a3-762b-4aee-9419-e909dc357ada",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ChatGLMForConditionalGeneration(\n",
       "  (transformer): ChatGLMModel(\n",
       "    (embedding): Embedding(\n",
       "      (word_embeddings): Embedding(65024, 4096)\n",
       "    )\n",
       "    (rotary_pos_emb): RotaryEmbedding()\n",
       "    (encoder): GLMTransformer(\n",
       "      (layers): ModuleList(\n",
       "        (0-27): 28 x GLMBlock(\n",
       "          (input_layernorm): RMSNorm()\n",
       "          (self_attention): SelfAttention(\n",
       "            (query_key_value): Linear4bit(in_features=4096, out_features=4608, bias=True)\n",
       "            (core_attention): CoreAttention(\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (dense): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
       "          )\n",
       "          (post_attention_layernorm): RMSNorm()\n",
       "          (mlp): MLP(\n",
       "            (dense_h_to_4h): Linear4bit(in_features=4096, out_features=27392, bias=False)\n",
       "            (dense_4h_to_h): Linear4bit(in_features=13696, out_features=4096, bias=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (final_layernorm): RMSNorm()\n",
       "    )\n",
       "    (output_layer): Linear(in_features=4096, out_features=65024, bias=False)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.requires_grad_(False)\n",
    "base_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6761d631-2cb8-4c2f-9dea-5edc45c2fd57",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_name_or_path,\n",
    "                                          trust_remote_code=True,\n",
    "                                          revision='b098244')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "eeb22447-f25f-4da2-8001-8bfd9f8785d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "response, history = base_model.chat(tokenizer, query=input_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9a628557-0ff6-477b-af76-7f2e31c48e98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "乾卦是八卦之一，也是八宫图说、易经、易学中的重要符号。乾卦的含义为天、为刚、为健。乾卦是由两个阴爻夹一个阳爻构成，象征天。乾卦的卦象是天宇的方形，由四个阳爻夹两个阴爻构成，表示天空中的云彩。乾卦象征着刚健、积极、进取、创造、领导等特质。在易经中，乾卦的卦辞为“元、亨、利、贞”，表示乾卦具有唯一性和正统性，能够通行无阻，达到自己的目的，是有利、正派、永恒的。乾卦的五行属性为木，代表春季，万物生长，充满生机。\n"
     ]
    }
   ],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "54d608cc-4c1f-48f7-a28c-c1b63f7a590b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "讼卦是八卦之一，也是八宫图说、易经、易学中的重要符号。讼卦的含义为决诉讼、解决争端。讼卦是由两个阳爻夹一个阴爻构成，象征法庭上的辩论和争端。讼卦的卦象是一个天平，表示公正、平等、公平。在易经中，讼卦的卦辞为“不利、诅咒、争斗、诉讼”，表示讼卦所象征的争端和诉讼往往是不利的，会带来诅咒和不幸。但是，如果能够以诚信、公平、正义的态度来处理争端，那么讼卦也可以表示通过公正的审判和调解，最终可以解决争端，恢复和平。\n"
     ]
    }
   ],
   "source": [
    "response, history = base_model.chat(tokenizer, query=\"周易中的讼卦是什么？\", history=history)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d731965-4a72-476a-a68e-e4761e7fddbd",
   "metadata": {},
   "source": [
    "# 遍历模型配置，准备使用微调后的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d6631791-78ca-4488-bed3-c1f0d2df853b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------\n",
      "使用微调后的第 1 个模型\n",
      "[gMASK]sop 解释下乾卦是什么？\n",
      "乾卦是八卦之一，也是八宫图说、易经、易学中最基本、最重要的卦象之一。乾卦为天、为刚、为健。乾卦是由两个阴爻夹一个阳爻构成，象征天、云、雷、震动等。乾卦的卦辞为“元、亨、利、贞”，表示天行健，君子以自强不息。乾卦的六爻分别代表不同的阳爻和阴爻，每一爻都有不同的含义和象征。\n",
      "\n",
      "乾卦的含义非常丰富，可以代表宇宙、自然、权力、刚强、运动、创造等。在易经和易学中，乾卦经常被用来象征父亲、领导、君主等权力人物，以及天空、云彩、雷声等自然现象。乾卦还可以代表阳刚之气、积极向上、刚强不倒等品质和精神。\n",
      "\n",
      "乾卦的六爻可以用来预测事物的发展和变化，也可以用来指导人们的行为和决策。在易经和易学中，乾卦的六爻有各自的阴阳属性，阴爻表示柔顺、阴暗、内敛等，阳爻表示刚强、明亮、外放等。通过观察乾卦的六爻，可以了解事物的变化趋势、发展方向和可能出现的问题。\n",
      "\n",
      "总之，乾卦是易经和易学中最基本、最重要的卦象之一，具有丰富的象征意义和预测功能。\n",
      "-------------------------\n",
      "使用微调后的第 2 个模型\n",
      "[gMASK]sop 解释下乾卦是什么？ 乾卦是周易中的一卦，代表天，具有刚健强劲的特性。它由六个阳爻组成，象征着天地的运行，以及阳刚之气。乾卦代表天象，以及由天象所代表的运行和力量，以及由这种力量带来的刚健强劲的特性。\n",
      "\n",
      "乾卦的初爻（卦首）为天，象征着刚健强劲的开始。随着卦象的展开，每一爻都为阳爻，表明了阳刚之力的不断增长。乾卦的最后一卦为天，再次强调天象和运行的重要性。\n",
      "\n",
      "乾卦的哲学含义是：天行刚健，代表阳刚之力的运行和力量，以及由这种力量带来的刚健强劲的特性。天地的运行无法停歇，就像阳刚之力的增长一样，无法阻挡。\n",
      "-------------------------\n",
      "使用微调后的第 3 个模型\n",
      "[gMASK]sop 解释下乾卦是什么？ 在周易中，乾卦是六十四卦之首，由六个阳爻组成，象征着天。它所代表的是刚健、健行、刚健不屈的意境。乾卦的核心哲学是：天道刚健，运行不已，君子观此卦象，从而以天为法，自强不息。\n",
      "\n",
      "乾卦象征天，为大通而至正。得此卦者，名利双收，应把握机会，争取成果。然而，切勿过于骄傲自满，而应保持谦逊、冷静和警惕。在事业、经商、求名等方面，乾卦皆暗示着大吉大利，但也警示着必须坚持正道、修养德行，方能永远亨通。\n",
      "\n",
      "在婚恋方面，乾卦提示着阳盛阴衰，但也强调刚柔相济，相互补足，形成美满的结果。在决策方面，则是强调刚健、正直、公允，自强不息的实质，需要修养德行、坚定信念，方能克服困难，消除灾难。\n"
     ]
    }
   ],
   "source": [
    "models=[1,2,3]\n",
    "from peft import PeftModel,PeftConfig\n",
    "for model_sn in models:\n",
    "    print(\"-\"*25)\n",
    "    print(\"使用微调后的第\",model_sn,\"个模型\")\n",
    "    model_dir=f\"models/{model_name_or_path}-{model_sn}\"\n",
    "    \n",
    "    quant_config=PeftConfig.from_pretrained(model_dir)\n",
    "    quant_model=PeftModel.from_pretrained(base_model,model_dir)\n",
    "    \n",
    "    inputs=tokenizer(input_text,return_tensors=\"pt\").to(0)\n",
    "    quant_out=quant_model.generate(**inputs,max_new_tokens=512)\n",
    "    quant_response=tokenizer.decode(quant_out[0],skip_special_tokens=True)\n",
    "    print(quant_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1379c07f-dc18-4fd9-9e49-567775deaea2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "完成使用微调后的模型\n"
     ]
    }
   ],
   "source": [
    "print(\"完成使用微调后的模型\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "278f8644-c071-4f72-8ea6-8e30b6245ebb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
